# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

import logging
import time
from fake_useragent import UserAgent
from scrapy.http import HtmlResponse
from tools.crawl_xici_ip import GetIP


logger = logging.getLogger(__name__)


class RandomUserAgentMiddleware(object):
    def __init__(self, crawler):
        super(RandomUserAgentMiddleware, self).__init__()

        self.ua = UserAgent()
        self.per_proxy = crawler.settings.get('RANDOM_UA_PER_PROXY', False)
        self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random')
        self.proxy2ua = {}

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def process_request(self, request, spider):
        def get_ua():
            '''Gets random UA based on the type setting (random, firefox…)'''
            return getattr(self.ua, self.ua_type)

        if self.per_proxy:
            proxy = request.meta.get('proxy')
            if proxy not in self.proxy2ua:
                self.proxy2ua[proxy] = get_ua()
                logger.debug('Assign User-Agent %s to Proxy %s'
                             % (self.proxy2ua[proxy], proxy))
            request.headers.setdefault('User-Agent', self.proxy2ua[proxy])
        else:
            ua = get_ua()
            request.headers.setdefault('User-Agent', ua)


class RandomProxyMiddleware(object):

    def process_request(self, request, spider):
        # get_ip = GetIP()
        # proxy = get_ip.get_random_ip()
        request.meta["proxy"] = 'http://118.190.95.35:9001'


class JspageMiddleware(object):

    #通过Chrome方式请求
    def process_request(self,request,spider):
        if spider.name == 'jobbole':
            # browser = webdriver.Chrome()
            spider.browser.get(request.url)
            time.sleep(3)
            print("访问：{}".format(request.url))
            return HtmlResponse(url=spider.browser.current_url,body=spider.page_source,encoding='utf-8',request=request)
#windows下有问题，在linux下试一试

# sudo apt-get install xvfb
# pip install xvfbwrapper

# from pyvirtualdisplay import Display
# from selenium import webdriver
#
# display = Display(visible=0, size=(800, 600))
# display.start()
#
# browser = webdriver.Chrome()
# browser.get('https://www.taobao.com')

